package test;



import java.io.IOException;
import java.text.BreakIterator;
import java.util.Iterator;
import java.util.Locale;
import org.apache.lucene.analysis.cn.smart.hhmm.SegToken;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.apache.lucene.analysis.util.SegmentingTokenizerBase;
import org.apache.lucene.util.AttributeFactory;

public class MyChinese extends SegmentingTokenizerBase {
	private static final BreakIterator sentenceProto;
	private final CharTermAttribute termAtt;
	private final OffsetAttribute offsetAtt;
	private final TypeAttribute typeAtt;
	private final WordSegmenter wordSegmenter;
	private Iterator<SegToken> tokens;

	public MyChinese() {
		this(DEFAULT_TOKEN_ATTRIBUTE_FACTORY);
	}

	public MyChinese(AttributeFactory factory) {
		super(factory, (BreakIterator) sentenceProto.clone());
		this.termAtt = (CharTermAttribute) this.addAttribute(CharTermAttribute.class);
		this.offsetAtt = (OffsetAttribute) this.addAttribute(OffsetAttribute.class);
		this.typeAtt = (TypeAttribute) this.addAttribute(TypeAttribute.class);
		this.wordSegmenter = new WordSegmenter();
	}

	protected void setNextSentence(int sentenceStart, int sentenceEnd) {
		String sentence = new String(this.buffer, sentenceStart, sentenceEnd - sentenceStart);
		this.tokens = this.wordSegmenter.segmentSentence(sentence, this.offset + sentenceStart).iterator();
	}

	protected boolean incrementWord() {
		if (this.tokens != null && this.tokens.hasNext()) {
			SegToken token = (SegToken) this.tokens.next();
			this.clearAttributes();
			this.termAtt.copyBuffer(token.charArray, 0, token.charArray.length);
			this.offsetAtt.setOffset(this.correctOffset(token.startOffset), this.correctOffset(token.endOffset));
			this.typeAtt.setType("word");
			return true;
		} else {
			return false;
		}
	}

	public void reset() throws IOException {
		super.reset();
		this.tokens = null;
	}

	static {
		sentenceProto = BreakIterator.getSentenceInstance(Locale.ROOT);
	}
}